package com.asiainfo.zqx;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.text.ParseException;
import java.text.SimpleDateFormat;

public class DongFangcaifuwang extends BreadthCrawler {
    private String staticseedurl="https://finance.eastmoney.com/";
    private  String activeseedurl="https://newsapi.eastmoney.com/kuaixun/v2/api/list?";
    public DongFangcaifuwang(String crawlPath) {
        super(crawlPath, false);
        addSeed(new CrawlDatum(staticseedurl,"list"));
        addSeed(new CrawlDatum(activeseedurl,"active"));
        setThreads(1);
       // setResumable(true);
    }

    @Override
    public void visit(Page page, CrawlDatums crawlDatums) {
        String contentType=page.contentType();
        if(contentType==null){
            return;
        }
        if(page.matchType("text")&&page.select("div.item").get(1).text().contains("东方财富网")){
            System.out.println("连接为:"+page.url());
            System.out.println("标题为:"+page.select("div.title").text());
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
            try {
                String text = page.select("div.item").first().text();
                if (text.contains("年")){
                    text = text.replace("年","-");
                }
                if (text.contains("月")){
                    text = text.replace("月","-");
                }
                if (text.contains("日")){
                    text = text.replace("日","");
                }
                System.out.println("text:"+text);
                System.out.println("时间为:"+sdf.parse(text));
            } catch (ParseException e) {
                throw new RuntimeException(e);
            }
            System.out.println("作者为:"+page.select("div.item").get(1).text().substring(3));
            System.out.println("来源为:东方财富网");
            System.out.println("正文为:"+page.select("div.txtinfos").text());
        }else if(page.matchType("list")&&page.url().equals("https://finance.eastmoney.com/")){
            Elements elements=page.select("a");
            for (Element element:elements) {
            String url=element.attr("abs:href");
                if(url.contains("finance.eastmoney.com/a/2022")&&url.matches("http://finance.eastmoney.com/a/.*.html")){
                    //将http改为https,否则网页无效
                    if(url.contains("http://")){
                        StringBuilder sb=new StringBuilder(url);
                        sb.insert(4,"s");
                        url=sb.toString();
                        System.out.println("静态链接为"+url);
                    }
                crawlDatums.add(new CrawlDatum(url,"text"));
            }}
        }else if(page.matchType("active")&&page.url().equals("https://newsapi.eastmoney.com/kuaixun/v2/api/list?")){
            String jsonStr=page.html();
            JsonParser parse =new JsonParser();  //创建json解析器
            JsonObject jsonObject=(JsonObject)parse.parse(jsonStr);  //创建jsonObject对象
            JsonArray arrayCards=jsonObject.get("news").getAsJsonArray();    //得到为json的数组
            //JsonArray array = arrayCards.get(0).getAsJsonObject().get("card_group").getAsJsonArray();
            for(int i=0;i<arrayCards.size();i++) {
                JsonObject subObject = arrayCards.get(i).getAsJsonObject();
                String geturl = subObject.get("url_unique").toString();
                if (geturl.contains("finance.eastmoney.com/a/2022")) {
                    if (geturl.contains("http://")) {
                        StringBuilder sb = new StringBuilder(geturl);
                        sb.insert(5, "s");
                        geturl = sb.substring(1,sb.length()-1);
                        System.out.println("动态链接为" + geturl);
                    }
                    crawlDatums.add(new CrawlDatum(geturl, "text"));
                }
            }
        }
    }

    public static void main(String[] args) throws Exception {
        DongFangcaifuwang dongf=new DongFangcaifuwang("dongfang");
        dongf.start(2);
    }
}
